{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 将少数类均SMOTE到（数据集样本数/类数）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "f:\\anaconda\\envs\\py37\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:526: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "f:\\anaconda\\envs\\py37\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:527: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "f:\\anaconda\\envs\\py37\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:528: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "f:\\anaconda\\envs\\py37\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:529: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "f:\\anaconda\\envs\\py37\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:530: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "f:\\anaconda\\envs\\py37\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:535: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "from collections import Counter\n",
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "\n",
    "data = np.load('E:/IDS/alldata/12/train/data.npy')\n",
    "label = np.load('E:/IDS/alldata/12/train/label_10.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "X=np.array(data)  #np.array将列表转换为数组，即去掉每个元素外面的中括号\n",
    "b=np.array(label)\n",
    "bb=b.reshape(b.shape[0],)     #reshape重新定义形状,此时标签数据是1维的\n",
    "y10 = np.int32(bb)  #将标签类型从浮点型换成整形"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1778030, 12)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0, 1553132),\n",
       " (1, 1874),\n",
       " (2, 1630),\n",
       " (3, 11449),\n",
       " (4, 31167),\n",
       " (5, 16972),\n",
       " (6, 150836),\n",
       " (7, 9791),\n",
       " (8, 1057),\n",
       " (9, 122)]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sorted(Counter(y10).items())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(0, 1553132), (1, 177803), (2, 177803), (3, 177803), (4, 177803), (5, 177803), (6, 177803), (7, 177803), (8, 177803), (9, 177803)]\n",
      "time: 143.16974186897278\n"
     ]
    }
   ],
   "source": [
    "#使用imlbearn库中上采样方法中的SMOTE接口\n",
    "from imblearn.over_sampling import SMOTE\n",
    "import time\n",
    "time_start = time.time()\n",
    "\n",
    "smo = SMOTE(ratio={1:177803,2:177803,3:177803,4:177803,5:177803,6:177803,7:177803,8:177803,9:177803},random_state=42)  #可以用ratio指定采样程度\n",
    "\n",
    "X_smo, y_smo = smo.fit_sample(X, y10)    #对数据和标签进行SMOTE处理\n",
    "print(sorted(Counter(y_smo).items()))\n",
    "\n",
    "time_end = time.time()\n",
    "time = time_end - time_start\n",
    "print(\"time:\",time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3153359"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_smo.shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 将多数类数据提取出来"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "list0 = []  #正常类数据\n",
    "list1 = []  #攻击类数据\n",
    "list2 = []  #攻击类标签\n",
    "\n",
    "for i in range(X_smo.shape[0]):\n",
    "    if y_smo[i] == 0:\n",
    "        list0.append(X_smo[i])\n",
    "    else:\n",
    "        list1.append(X_smo[i])\n",
    "        list2.append(y_smo[i])\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "正常类数据形状： (1553132, 12)\n",
      "攻击类数据形状： (1600227, 12)\n",
      "攻击类标签形状： (1600227,)\n"
     ]
    }
   ],
   "source": [
    "data0 = np.array(list0)  #np.array将列表转换为数组，即去掉每个元素外面的中括号\n",
    "data1 = np.array(list1)\n",
    "label1 = np.array(list2)\n",
    "\n",
    "label11 = label1.reshape(label1.shape[0],)     #reshape重新定义形状,此时标签数据是1维的\n",
    "\n",
    "print(\"正常类数据形状：\",data0.shape)\n",
    "print(\"攻击类数据形状：\",data1.shape)  #使用函数查看类分布需要这种格式\n",
    "print(\"攻击类标签形状：\",label11.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5, 6, 6, ..., 9, 9, 9])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label11"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 将多数类数据聚成（总类数）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.mixture import GaussianMixture\n",
    "import time\n",
    "time_start = time.time()\n",
    "\n",
    "estimator = GaussianMixture(n_components=10)\n",
    "estimator.fit(data0)  # 聚类\n",
    "\n",
    "time_end = time.time()\n",
    "time = time_end - time_start\n",
    "print(\"time:\",time)\n",
    "\n",
    "label_pred = estimator.predict(data0)  # 获取聚类标签"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 49.27290630340576\n"
     ]
    }
   ],
   "source": [
    "from sklearn.cluster import KMeans\n",
    "import time\n",
    "time_start = time.time()\n",
    "\n",
    "estimator = KMeans(n_clusters=10)\n",
    "estimator.fit(data0)  # 聚类\n",
    "\n",
    "time_end = time.time()\n",
    "time = time_end - time_start\n",
    "print(\"time:\",time)\n",
    "\n",
    "label_pred = estimator.labels_  # 获取聚类标签"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0, 192248),\n",
       " (1, 449896),\n",
       " (2, 277310),\n",
       " (3, 131107),\n",
       " (4, 27652),\n",
       " (5, 143659),\n",
       " (6, 229698),\n",
       " (7, 7405),\n",
       " (8, 92530),\n",
       " (9, 1627)]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sorted(Counter(label_pred).items())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 从每个簇中选出一定量的数据组成新的多数类数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "c0 = []\n",
    "c1 = []\n",
    "c2 = []\n",
    "c3 = []\n",
    "c4 = []\n",
    "c5 = []\n",
    "c6 = []\n",
    "c7 = []\n",
    "c8 = []\n",
    "c9 = []\n",
    "\n",
    "\n",
    "s0=s1=s2=s3=s4=s5=s6=s7=s8=s9=0\n",
    "\n",
    "for i in range(data0.shape[0]):\n",
    "    if label_pred[i] == 0:\n",
    "        c0.append(data0[i])\n",
    "        s0=s0+1\n",
    "    elif label_pred[i] == 1:\n",
    "        c1.append(data0[i])\n",
    "        s1=s1+1\n",
    "    elif label_pred[i] == 2:\n",
    "        c2.append(data0[i])\n",
    "        s2=s2+1\n",
    "    elif label_pred[i] == 3:\n",
    "        c3.append(data0[i])\n",
    "        s3=s3+1\n",
    "    elif label_pred[i] == 4:\n",
    "        c4.append(data0[i])\n",
    "        s4=s4+1\n",
    "    elif label_pred[i] == 5:\n",
    "        c5.append(data0[i])\n",
    "        s5=s5+1\n",
    "    elif label_pred[i] == 6:\n",
    "        c6.append(data0[i])\n",
    "        s6=s6+1\n",
    "    elif label_pred[i] == 7:\n",
    "        c7.append(data0[i])\n",
    "        s7=s7+1\n",
    "    elif label_pred[i] == 8:\n",
    "        c8.append(data0[i])\n",
    "        s8=s8+1\n",
    "    elif label_pred[i] == 9:\n",
    "        c9.append(data0[i])\n",
    "        s9=s9+1\n",
    "        \n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = 21096"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "删除前大小: 192248\n",
      "删除后大小: 21096\n"
     ]
    }
   ],
   "source": [
    "print(\"删除前大小:\",len(c0))\n",
    "del c0[a:len(c0)]\n",
    "print(\"删除后大小:\",len(c0))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "删除前大小: 449896\n",
      "删除后大小: 21096\n"
     ]
    }
   ],
   "source": [
    "print(\"删除前大小:\",len(c1))\n",
    "del c1[a:len(c1)]\n",
    "print(\"删除后大小:\",len(c1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "删除前大小: 277310\n",
      "删除后大小: 21096\n"
     ]
    }
   ],
   "source": [
    "print(\"删除前大小:\",len(c2))\n",
    "del c2[a:len(c2)]\n",
    "print(\"删除后大小:\",len(c2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "删除前大小: 131107\n",
      "删除后大小: 21096\n"
     ]
    }
   ],
   "source": [
    "print(\"删除前大小:\",len(c3))\n",
    "del c3[a:len(c3)]\n",
    "print(\"删除后大小:\",len(c3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "删除前大小: 27652\n",
      "删除后大小: 21096\n"
     ]
    }
   ],
   "source": [
    "print(\"删除前大小:\",len(c4))\n",
    "del c4[a:len(c4)]\n",
    "print(\"删除后大小:\",len(c4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "del c5[a:len(c5)]\n",
    "del c6[a:len(c6)]\n",
    "del c7[a:len(c7)]\n",
    "del c8[a:len(c8)]\n",
    "del c9[a:len(c9)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "c00 = np.array(c0)\n",
    "c11 = np.array(c1)\n",
    "c22 = np.array(c2)\n",
    "c33 = np.array(c3)\n",
    "c44 = np.array(c4)\n",
    "c55 = np.array(c5)\n",
    "c66 = np.array(c6)\n",
    "c77 = np.array(c7)\n",
    "c88 = np.array(c8)\n",
    "c99 = np.array(c9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(177800, 12)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q = np.concatenate((c00,c11,c22,c33,c44,c55,c66,c77,c88,c99),axis=0)\n",
    "q.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(177800,)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#创建一个全零数组为正常类的标签\n",
    "label_zc = np.zeros((q.shape[0],), dtype=int)\n",
    "label_zc.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "#将正常类数据和攻击数据合并到一起，将标签也合并到一起\n",
    "data_end = np.concatenate((q,data1),axis=0)\n",
    "label_end = np.concatenate((label_zc,label1),axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0, 177800),\n",
       " (1, 177803),\n",
       " (2, 177803),\n",
       " (3, 177803),\n",
       " (4, 177803),\n",
       " (5, 177803),\n",
       " (6, 177803),\n",
       " (7, 177803),\n",
       " (8, 177803),\n",
       " (9, 177803)]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sorted(Counter(label_end).items())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_end = label_end.reshape(label_end.shape[0],1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "#将最终采样后的数据保存成文件\n",
    "np.save(\"E:/IDS/alldata/12/k-means+smote_train/data.npy\",data_end)\n",
    "np.save(\"E:/IDS/alldata/12/k-means+smote_train/label_10.npy\",label_end)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
